package Interview.Amazon;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.Set;

import org.apache.commons.compress.utils.Lists;

/**
 * @author leiwenkang <leiwenkang@kuaishou.com>
 * Created on 2022-08-07
 */
public class Test07网络爬虫 {

    public static void main(String[] args) {
        System.out.println(dfsCrawl("http://news.yahoo.com/news/topics/", new HtmlParser()));
    }

    //BFS
    public static List<String> bfsCrawl(String startUrl, HtmlParser htmlParser) {
        Set<String> urlSet = new HashSet<>();
        //BFS
        Queue<String> queue = new LinkedList<>();
        queue.add(startUrl);
        urlSet.add(startUrl);
        String hostName = getHostName(startUrl);

        while (!queue.isEmpty()) {
            String url = queue.poll();

            List<String> crawlUrls = htmlParser.getUrls(url);
            for (String crawlUrl : crawlUrls) {
                if (!urlSet.contains(crawlUrl) && hostName.equals(getHostName(crawlUrl))) {
                    queue.add(crawlUrl);
                    urlSet.add(crawlUrl);
                }
            }
        }
        return new ArrayList<>(urlSet);

    }

    //DFS
    public static List<String> dfsCrawl(String startUrl, HtmlParser htmlParser) {
        Set<String> urlSet = new HashSet<>();
        dfs(startUrl, htmlParser, urlSet);
        return new ArrayList<>(urlSet);
    }

    //DFS
    public static void dfs(String startUrl, HtmlParser htmlParser, Set<String> urlSet) {
        urlSet.add(startUrl);
        String hostName = getHostName(startUrl);

        List<String> crawlUrls = htmlParser.getUrls(startUrl);
        for (String crawlUrl : crawlUrls) {
            if (!urlSet.contains(crawlUrl) && hostName.equals(getHostName(crawlUrl))) {
                urlSet.add(crawlUrl);
                dfs(crawlUrl, htmlParser, urlSet);
            }
        }
    }

    private static String getHostName(String url) {
        String hostName = url.substring(7);
        if (hostName.contains("/")) {
            int indexOfFirstLine = hostName.indexOf('/');
            hostName = hostName.substring(0, indexOfFirstLine);
        }
        return hostName;
    }

}

class HtmlParser {
    public List<String> getUrls(String url) {
        List<String> list = new ArrayList<>();
        list.add("http://news.yahoo.com");
        list.add("http://news.yahoo.com/news");
        list.add("http://news.yahoo.com/news/topics/");
        list.add("http://news.google.com");
        list.add("http://news.yahoo.com/us");
        return list;
    }
}
